#!/usr/bin/python
#-*-coding:utf-8-*-
# 简易小说采集爬虫
# Author: Lukin<mylukin@gmail.com>
# Date  : 2008-09-25
import re
import sqlite3
import os.path
import string
from download import httpfetch
basedb = './book.db'
urlbase = 'http://www.book8.com/'
#<td class="odd"><a href="/Bookinfo/[0-9]+/[0-9]+.htm">.+</a>
rule_list = r'<td class="odd"><a href="/Bookinfo/[0-9]+/([0-9]+).htm">(.+)</a></td> <td class="even">'
conn = sqlite3.connect(os.path.abspath(basedb))
conn.text_factory = str
c = conn.cursor()

def createDatabase():
    # 创建url列表存放表
    c.execute('SELECT * FROM sqlite_master WHERE name = \'BookList\';')
    if c.fetchall():
        c.execute('DROP TABLE \'BookList\'')
        conn.commit()
    c.execute('''CREATE TABLE "BookList" ("id" INTEGER PRIMARY KEY ,"Bid" INTEGER,"Title" TEXT(60),"Author" TEXT(60),"clong" INTEGER,"lasttime" TEXT,"lastname" TEXT,"lastid" INTEGER,"hottxt" TEXT,"isupdate" INTEGER DEFAULT 0 ,"type" TEXT,"pic" TEXT)''')
    # 事物提交
    conn.commit()
    
def read_list(page):
    url = urlbase + 'Booktoppostdate/0/' + page + '.htm'
    headers = {"Accept": "*/*", "Referer": "http://www.book8.com/", "User-Agent": "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)"}
    req = httpfetch(url,headers)
    return req.decode('gbk','ignore').encode('utf-8')

def parse_list(html):
    names = re.findall(rule_list,html)
    return names

def clear(str):
    str = string.replace(str,'amp;','')
    str = string.replace(str,'&nbsp;','')
    str = string.replace(str,'&gt;','')
    str = string.replace(str,'&lt;','')
    str = string.replace(str,'&lt;','')
    return str


def data_store(names):
    for name in names:
        print name[0] + '::::::' + name[1]
        c.execute('SELECT * FROM BookList WHERE Bid = ?;',(name[0],))
        i = c.fetchall()
        if i == None:
            print name[0]
        else:
            c.execute('INSERT INTO BookList (Bid,Title) VALUES (?,?);',(name[0],clear(name[1])))
            print '成功入库:'.decode('utf-8','ignore').encode('gbk') + name[1].decode('utf-8','ignore').encode('gbk')
    conn.commit()
        
createDatabase()
for i in range(298):
    data_store(parse_list(read_list(str(i + 1))))    
    print '成功采集第'.decode('utf-8','ignore').encode('gbk') + str(i + 1) + '页'.decode('utf-8','ignore').encode('gbk')

c.close()